import sys,os,time,re
from bs4 import BeautifulSoup

filename = sys.argv[1]
dst_dir = sys.argv[2]

max_length = 1024*1024*1024*10
max_length = 1024*1024*1024
print filename, dst_dir

#-------------------#
def del_empty_line(text):
    return re.sub(r'\n[\s| ]*\n', '', text.strip())
#---------#
def mkdirs(path):
    try:
        os.makedirs(path)
    except:
        pass
#---------#
def newfile(filename, content):
    fp = open(filename, 'w')
    fp.write(content)
    fp.close()
#---------#
def get_strings(soup):
    text = ''
    if not soup:
        return text
    for string in soup.stripped_strings:
        text += (string + '\n')
    return text
#---------#
def get_content(soup):
    #reg = re.compile("<[^>]*>")
    #content = reg.sub('',soup.prettify())
    text = del_empty_line(get_strings(soup))
    try:
        return text.encode('utf8')
    except:
        return text
#---------#
def get_title(soup):
    title = ''
    if soup.title and soup.title.string:
        try:
            title = soup.title.string.encode('utf8')
        except:
            title = soup.title.string
    return del_empty_line(title)
#---------#
def set_cfg(filenamenum):
    fp = open("config.cfg", 'w')
    fp.write(str(filenamenum))
    fp.close()
#---------#
def get_cfg():
    try:
        fp = open("config.cfg", 'r')
        filenamenum=fp.readline()
        fp.close()
        return int(filenamenum)
    except:
        set_cfg(1)
        return 1
#---------#
def get_info(filename):
    fp = open(filename)
    data = fp.read()
    fp.close()

    tmp = data.split('\n', 3)
    
    soup = BeautifulSoup(tmp[3], "lxml")
    [x.extract() for x in soup.find_all('script')]
    [style.extract() for style in soup.find_all('style')]

    return {
        "url": tmp[0],
        "title": get_title(soup),
        "docno": tmp[2],
        "content": get_content(soup)
    }
#---------#
def save_info(src, dst):
    info = get_info(src)
    fp = open(dst, 'a')
    fp.write("\n_tgtxt_page_\n\n")
    fp.write("sublib:sougou\n")
    fp.write("title:"+info["title"]+"\n")
    fp.write("url:"+info["url"]+"\n")
    fp.write(info["content"]+"\n")
    fp.close()
#---------#
def get_dst_file():
    fnum = get_cfg()
    file = os.path.join(dst_dir, str(fnum)+".tgtxt")
    try:
        size = os.path.getsize(file)
        if size < max_length:
            return file
        fnum += 1
        set_cfg(fnum)
        return os.path.join(dst_dir, str(fnum)+".tgtxt")
    except:
        return file
#-------------------#

#-------------------#
mkdirs(dst_dir)
#try:
save_info(filename, get_dst_file())
sys.exit(0)
#except:
#sys.exit(1)
#-------------------#
